fix char string corruption with xcsv. (#455)

Any gmsd char string data handled by the xcsv format could be
corrupted when main ran cet_convert_strings.  The xml format
reader/writer use a QTextCodec to handle all char string conversion,
but the data was converted again by cet_convert_strings ... garmin_fs_convert
to/from ASCII, resulting in corruption.

Add a test case for the above.

Add xcsv format support for gmsd email field.

diff --git a/reference/gmsd.unicsv b/reference/gmsd.unicsv
new file mode 100644 (file)
index 0000000..1422bd5
--- /dev/null
+++ b/reference/gmsd.unicsv
@@ -0,0 +1,3 @@
+No,Latitude,Longitude,Name,Description,URL,Address,City,PostalCode,Country,Phone,Email\r
+1,52.523013,13.398808,"MA'LOA Poké Bowl","restaurant","https://www.maloa.com/","Oranienburger Straße 7","Berlin","10178","Deutschland","030-28427238","franchise@maloa.com"\r
+2,55.741225,37.524863,"Verde café","ресторан","http://verde-cafe.ru/","дом, Kutuzovsky Ave, 36, строение 4, подъезд 3","Москва","121170","Россия","+7 499 350-96-44","info@green-company.org"\r

diff --git a/reference/gmsd.xcsv b/reference/gmsd.xcsv
new file mode 100644 (file)
index 0000000..406c904
--- /dev/null
+++ b/reference/gmsd.xcsv
@@ -0,0 +1,2 @@
+52.523013      13.398808       MA'LOA Poké Bowl       restaurant      https://www.maloa.com/  Oranienburger Straße 7 Berlin  Deutschland     030-28427238    10178   franchise@maloa.com
+55.741225      37.524863       Verde café     ресторан        http://verde-cafe.ru/   дом, Kutuzovsky Ave, 36, строение 4, подъезд 3        Москва    Россия    +7 499 350-96-44        121170  info@green-company.org

diff --git a/testo.d/xcsv.test b/testo.d/xcsv.test
index 6cb0569a67f04eb07afe2ed857f5736c37283e3c..7fe41655323a2f839ed60ce11413a0dd4167f7e5 100644 (file)
--- a/testo.d/xcsv.test
+++ b/testo.d/xcsv.test
@@ -70,3 +70,25 @@ echo 'IFIELD LON_DECIMAL,"","%f"' >>${TMPDIR}/route1.style
 echo 'IFIELD ROUTE_NAME,"","%s"' >>${TMPDIR}/route1.style
 gpsbabel -i xcsv,style=${TMPDIR}/route1.style -f ${REFERENCE}/route/route1.csv -o gpx -F ${TMPDIR}/route1~csv.gpx
 compare ${REFERENCE}/route/route1~csv.gpx ${TMPDIR}/route1~csv.gpx
+
+# gmsd fields
+echo 'DESCRIPTION gmsd test' >> ${TMPDIR}/gmsd.style
+echo 'EXTENSION csv' >> ${TMPDIR}/gmsd.style
+echo 'FIELD_DELIMITER TAB' >> ${TMPDIR}/gmsd.style
+echo 'RECORD_DELIMITER NEWLINE' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD LAT_DECIMAL,"","%f"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD LON_DECIMAL,"","%f"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD SHORTNAME,"","%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD DESCRIPTION,"","%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD URL,"","%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD STREET_ADDR, "", "%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD CITY, "", "%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD COUNTRY, "", "%s"' >> ${TMPDIR}/gmsd.style
+#echo 'IFIELD FACILITY, "", "%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD PHONE_NR, "", "%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD POSTAL_CODE, "", "%s"' >> ${TMPDIR}/gmsd.style
+echo 'IFIELD EMAIL, "", "%s"' >> ${TMPDIR}/gmsd.style
+gpsbabel -i unicsv -f ${REFERENCE}/gmsd.unicsv -o xcsv,style=${TMPDIR}/gmsd.style -F ${TMPDIR}/gmsd.xcsv
+compare ${REFERENCE}/gmsd.xcsv ${TMPDIR}/gmsd.xcsv
+gpsbabel -i xcsv,style=${TMPDIR}/gmsd.style -f ${REFERENCE}/gmsd.xcsv -o unicsv -F ${TMPDIR}/gmsd.unicsv
+compare ${REFERENCE}/gmsd.unicsv ${TMPDIR}/gmsd.unicsv

diff --git a/xcsv.cc b/xcsv.cc
index e6627c3ac23da20a9d4f336d8e7cc3bc69aab791..dadb466b5211cb9e33e78f39e5dba0020772c6ed 100644 (file)
--- a/xcsv.cc
+++ b/xcsv.cc
@@ -93,6 +93,7 @@ enum xcsv_token {
   XT_CONSTANT,
   XT_COUNTRY,
   XT_DESCRIPTION,
+  XT_EMAIL,
   XT_EXCEL_TIME,
   XT_FACILITY,
   XT_FILENAME,
@@ -982,6 +983,11 @@ xcsv_parse_val(const char* s, Waypoint* wpt, const field_map& fmp,
     GMSD_SET(facility, csv_stringtrim(s, enclosure, 0));
   }
   break;
+  case XT_EMAIL: {
+    garmin_fs_t* gmsd = gmsd_init(wpt);
+    GMSD_SET(email, csv_stringtrim(s, enclosure, 0));
+  }
+  break;
   case -1:
     if (strncmp(fmp.key.constData(), "LON_10E", 7) == 0) {
       wpt->longitude = atof(s) / pow(10.0, atof(fmp.key.constData()+7));
@@ -1705,6 +1711,10 @@ xcsv_waypt_pr(const Waypoint* wpt)
       garmin_fs_t* gmsd = GMSD_FIND(wpt);
       buff = QString().sprintf(fmp.printfc.constData(), GMSD_GET(facility, ""));
     }
+    case XT_EMAIL: {
+      garmin_fs_t* gmsd = GMSD_FIND(wpt);
+      buff = QString().sprintf(fmp.printfc.constData(), GMSD_GET(email, ""));
+    }
     break;
     /* specials */
     case XT_FILENAME:
@@ -2224,7 +2234,7 @@ ff_vecs_t xcsv_vecs = {
   xcsv_data_write,
   nullptr,
   &xcsv_args,
-  CET_CHARSET_ASCII, 0,        /* CET-REVIEW */
+  CET_CHARSET_UTF8, 0, /* conversion to utf8 for gmsd is handled by the stream, don't let csv_convert_strings convert gmsd data */
   { nullptr, nullptr, nullptr, xcsv_wr_position_init, xcsv_wr_position, xcsv_wr_position_deinit },
   nullptr
 

diff --git a/xcsv_tokens.gperf b/xcsv_tokens.gperf
index b2fabcbf7e563ea411b59fd31028d4bfdf081a3c..235530e2c726a6baca1eff20143daa7c19c0f92a 100644 (file)
--- a/xcsv_tokens.gperf
+++ b/xcsv_tokens.gperf
@@ -32,7 +32,7 @@
 #line 1 "xcsv_tokens.in"
 struct xt_mapping {const char *name; int xt_token; };
 
-#define TOTAL_KEYWORDS 86
+#define TOTAL_KEYWORDS 87
 #define MIN_WORD_LENGTH 3
 #define MAX_WORD_LENGTH 21
 #define MIN_HASH_VALUE 7
@@ -111,177 +111,179 @@ Perfect_Hash::in_word_set (const char *str, size_t len)
 {
   static struct xt_mapping wordlist[] =
     {
-#line 41 "xcsv_tokens.in"
+#line 42 "xcsv_tokens.in"
       {"LAT_DIR", XT_LAT_DIR},
-#line 69 "xcsv_tokens.in"
+#line 70 "xcsv_tokens.in"
       {"POWER", XT_POWER},
-#line 45 "xcsv_tokens.in"
+#line 46 "xcsv_tokens.in"
       {"LAT_DDMMDIR", XT_LAT_DDMMDIR},
-#line 51 "xcsv_tokens.in"
+#line 52 "xcsv_tokens.in"
       {"LON_DIR", XT_LON_DIR},
-#line 38 "xcsv_tokens.in"
+#line 39 "xcsv_tokens.in"
       {"LAT_DECIMALDIR", XT_LAT_DECIMALDIR},
-#line 54 "xcsv_tokens.in"
+#line 55 "xcsv_tokens.in"
       {"LON_DDMMDIR", XT_LON_DDMMDIR},
-#line 48 "xcsv_tokens.in"
+#line 49 "xcsv_tokens.in"
       {"LON_DECIMALDIR", XT_LON_DECIMALDIR},
-#line 58 "xcsv_tokens.in"
+#line 59 "xcsv_tokens.in"
       {"NET_TIME", XT_NET_TIME},
-#line 30 "xcsv_tokens.in"
+#line 31 "xcsv_tokens.in"
       {"HEART_RATE", XT_HEART_RATE},
-#line 73 "xcsv_tokens.in"
+#line 74 "xcsv_tokens.in"
       {"STREET_ADDR", XT_STREET_ADDR},
 #line 6 "xcsv_tokens.in"
       {"CADENCE", XT_CADENCE},
-#line 46 "xcsv_tokens.in"
+#line 47 "xcsv_tokens.in"
       {"LAT_NMEA", XT_LAT_NMEA},
-#line 70 "xcsv_tokens.in"
+#line 71 "xcsv_tokens.in"
       {"ROUTE_NAME", XT_ROUTE_NAME},
-#line 68 "xcsv_tokens.in"
+#line 69 "xcsv_tokens.in"
       {"POSTAL_CODE", XT_POSTAL_CODE},
-#line 55 "xcsv_tokens.in"
+#line 56 "xcsv_tokens.in"
       {"LON_NMEA", XT_LON_NMEA},
-#line 72 "xcsv_tokens.in"
+#line 73 "xcsv_tokens.in"
       {"STATE", XT_STATE},
-#line 22 "xcsv_tokens.in"
+#line 23 "xcsv_tokens.in"
       {"GEOCACHE_TERR", XT_GEOCACHE_TERR},
-#line 21 "xcsv_tokens.in"
+#line 22 "xcsv_tokens.in"
       {"GEOCACHE_PLACER", XT_GEOCACHE_PLACER},
-#line 82 "xcsv_tokens.in"
+#line 83 "xcsv_tokens.in"
       {"UTM", XT_UTM},
-#line 18 "xcsv_tokens.in"
-      {"GEOCACHE_ISARCHIVED", XT_GEOCACHE_ISARCHIVED},
 #line 19 "xcsv_tokens.in"
+      {"GEOCACHE_ISARCHIVED", XT_GEOCACHE_ISARCHIVED},
+#line 20 "xcsv_tokens.in"
       {"GEOCACHE_ISAVAILABLE", XT_GEOCACHE_ISAVAILABLE},
 #line 5 "xcsv_tokens.in"
       {"ANYNAME", XT_ANYNAME},
-#line 24 "xcsv_tokens.in"
+#line 25 "xcsv_tokens.in"
       {"GMT_TIME", XT_GMT_TIME},
-#line 76 "xcsv_tokens.in"
+#line 77 "xcsv_tokens.in"
       {"TIMET_TIME", XT_TIMET_TIME},
-#line 14 "xcsv_tokens.in"
+#line 15 "xcsv_tokens.in"
       {"FORMAT", XT_FORMAT},
-#line 13 "xcsv_tokens.in"
+#line 14 "xcsv_tokens.in"
       {"FILENAME", XT_FILENAME},
-#line 33 "xcsv_tokens.in"
+#line 34 "xcsv_tokens.in"
       {"ICON_DESCR", XT_ICON_DESCR},
 #line 10 "xcsv_tokens.in"
       {"DESCRIPTION", XT_DESCRIPTION},
-#line 16 "xcsv_tokens.in"
+#line 17 "xcsv_tokens.in"
       {"GEOCACHE_DIFF", XT_GEOCACHE_DIFF},
-#line 15 "xcsv_tokens.in"
+#line 16 "xcsv_tokens.in"
       {"GEOCACHE_CONTAINER", XT_GEOCACHE_CONTAINER},
-#line 79 "xcsv_tokens.in"
+#line 80 "xcsv_tokens.in"
       {"TRACK_NEW", XT_TRACK_NEW},
-#line 78 "xcsv_tokens.in"
+#line 79 "xcsv_tokens.in"
       {"TRACK_NAME", XT_TRACK_NAME},
-#line 81 "xcsv_tokens.in"
+#line 82 "xcsv_tokens.in"
       {"URL", XT_URL},
-#line 66 "xcsv_tokens.in"
+#line 67 "xcsv_tokens.in"
       {"PATH_SPEED", XT_PATH_SPEED},
-#line 59 "xcsv_tokens.in"
+#line 60 "xcsv_tokens.in"
       {"PATH_COURSE", XT_PATH_COURSE},
-#line 17 "xcsv_tokens.in"
+#line 18 "xcsv_tokens.in"
       {"GEOCACHE_HINT", XT_GEOCACHE_HINT},
-#line 39 "xcsv_tokens.in"
+#line 40 "xcsv_tokens.in"
       {"LAT_DECIMAL", XT_LAT_DECIMAL},
 #line 9 "xcsv_tokens.in"
       {"COUNTRY", XT_COUNTRY},
-#line 42 "xcsv_tokens.in"
+#line 43 "xcsv_tokens.in"
       {"LAT_HUMAN_READABLE", XT_LAT_HUMAN_READABLE},
-#line 47 "xcsv_tokens.in"
+#line 48 "xcsv_tokens.in"
       {"LOCAL_TIME", XT_LOCAL_TIME},
-#line 49 "xcsv_tokens.in"
+#line 50 "xcsv_tokens.in"
       {"LON_DECIMAL", XT_LON_DECIMAL},
-#line 52 "xcsv_tokens.in"
+#line 53 "xcsv_tokens.in"
       {"LON_HUMAN_READABLE", XT_LON_HUMAN_READABLE},
-#line 71 "xcsv_tokens.in"
+#line 72 "xcsv_tokens.in"
       {"SHORTNAME", XT_SHORTNAME},
-#line 67 "xcsv_tokens.in"
+#line 68 "xcsv_tokens.in"
       {"PHONE_NR", XT_PHONE_NR},
 #line 7 "xcsv_tokens.in"
       {"CITY", XT_CITY},
-#line 74 "xcsv_tokens.in"
+#line 75 "xcsv_tokens.in"
       {"TEMPERATURE", XT_TEMPERATURE},
-#line 37 "xcsv_tokens.in"
+#line 38 "xcsv_tokens.in"
       {"ISO_TIME", XT_ISO_TIME},
 #line 3 "xcsv_tokens.in"
       {"ALT_FEET", XT_ALT_FEET},
-#line 32 "xcsv_tokens.in"
+#line 33 "xcsv_tokens.in"
       {"HMSL_TIME", XT_HMSL_TIME},
-#line 56 "xcsv_tokens.in"
+#line 57 "xcsv_tokens.in"
       {"MAP_EN_BNG", XT_MAP_EN_BNG},
-#line 34 "xcsv_tokens.in"
+#line 35 "xcsv_tokens.in"
       {"IGNORE", XT_IGNORE},
-#line 75 "xcsv_tokens.in"
+#line 76 "xcsv_tokens.in"
       {"TEMPERATURE_F", XT_TEMPERATURE_F},
-#line 40 "xcsv_tokens.in"
+#line 41 "xcsv_tokens.in"
       {"LAT_DIRDECIMAL", XT_LAT_DIRDECIMAL},
-#line 35 "xcsv_tokens.in"
+#line 36 "xcsv_tokens.in"
       {"INDEX", XT_INDEX},
-#line 83 "xcsv_tokens.in"
+#line 84 "xcsv_tokens.in"
       {"UTM_ZONE", XT_UTM_ZONE},
-#line 50 "xcsv_tokens.in"
+#line 51 "xcsv_tokens.in"
       {"LON_DIRDECIMAL", XT_LON_DIRDECIMAL},
-#line 44 "xcsv_tokens.in"
+#line 45 "xcsv_tokens.in"
       {"LATLON_HUMAN_READABLE", XT_LATLON_HUMAN_READABLE},
 #line 8 "xcsv_tokens.in"
       {"CONSTANT", XT_CONSTANT},
-#line 20 "xcsv_tokens.in"
+#line 21 "xcsv_tokens.in"
       {"GEOCACHE_LAST_FOUND", XT_GEOCACHE_LAST_FOUND},
-#line 11 "xcsv_tokens.in"
+#line 12 "xcsv_tokens.in"
       {"EXCEL_TIME", XT_EXCEL_TIME},
-#line 86 "xcsv_tokens.in"
+#line 87 "xcsv_tokens.in"
       {"UTM_EASTING", XT_UTM_EASTING},
-#line 23 "xcsv_tokens.in"
+#line 24 "xcsv_tokens.in"
       {"GEOCACHE_TYPE", XT_GEOCACHE_TYPE},
-#line 85 "xcsv_tokens.in"
+#line 86 "xcsv_tokens.in"
       {"UTM_ZONEF", XT_UTM_ZONEF},
-#line 88 "xcsv_tokens.in"
+#line 89 "xcsv_tokens.in"
       {"YYYYMMDD_TIME", XT_YYYYMMDD_TIME},
-#line 31 "xcsv_tokens.in"
+#line 32 "xcsv_tokens.in"
       {"HMSG_TIME", XT_HMSG_TIME},
-#line 60 "xcsv_tokens.in"
+#line 61 "xcsv_tokens.in"
       {"PATH_DISTANCE_KM", XT_PATH_DISTANCE_KM},
-#line 12 "xcsv_tokens.in"
+#line 13 "xcsv_tokens.in"
       {"FACILITY", XT_FACILITY},
-#line 80 "xcsv_tokens.in"
+#line 81 "xcsv_tokens.in"
       {"URL_LINK_TEXT", XT_URL_LINK_TEXT},
-#line 84 "xcsv_tokens.in"
+#line 85 "xcsv_tokens.in"
       {"UTM_ZONEC", XT_UTM_ZONEC},
-#line 25 "xcsv_tokens.in"
+#line 26 "xcsv_tokens.in"
       {"GPS_FIX", XT_GPS_FIX},
-#line 77 "xcsv_tokens.in"
+#line 78 "xcsv_tokens.in"
       {"TIMET_TIME_MS", XT_TIMET_TIME_MS},
-#line 57 "xcsv_tokens.in"
+#line 58 "xcsv_tokens.in"
       {"NOTES", XT_NOTES},
-#line 65 "xcsv_tokens.in"
+#line 66 "xcsv_tokens.in"
       {"PATH_SPEED_MPH", XT_PATH_SPEED_MPH},
-#line 64 "xcsv_tokens.in"
+#line 65 "xcsv_tokens.in"
       {"PATH_SPEED_KPH", XT_PATH_SPEED_KPH},
-#line 36 "xcsv_tokens.in"
+#line 37 "xcsv_tokens.in"
       {"ISO_TIME_MS", XT_ISO_TIME_MS},
-#line 29 "xcsv_tokens.in"
+#line 30 "xcsv_tokens.in"
       {"GPS_VDOP", XT_GPS_VDOP},
 #line 4 "xcsv_tokens.in"
       {"ALT_METERS", XT_ALT_METERS},
-#line 28 "xcsv_tokens.in"
+#line 29 "xcsv_tokens.in"
       {"GPS_SAT", XT_GPS_SAT},
-#line 62 "xcsv_tokens.in"
+#line 63 "xcsv_tokens.in"
       {"PATH_DISTANCE_MILES", XT_PATH_DISTANCE_MILES},
-#line 61 "xcsv_tokens.in"
+#line 62 "xcsv_tokens.in"
       {"PATH_DISTANCE_METERS", XT_PATH_DISTANCE_METERS},
-#line 63 "xcsv_tokens.in"
+#line 64 "xcsv_tokens.in"
       {"PATH_SPEED_KNOTS", XT_PATH_SPEED_KNOTS},
-#line 43 "xcsv_tokens.in"
+#line 11 "xcsv_tokens.in"
+      {"EMAIL", XT_EMAIL},
+#line 44 "xcsv_tokens.in"
       {"LAT_INT32DEG", XT_LAT_INT32DEG},
-#line 53 "xcsv_tokens.in"
+#line 54 "xcsv_tokens.in"
       {"LON_INT32DEG", XT_LON_INT32DEG},
-#line 87 "xcsv_tokens.in"
+#line 88 "xcsv_tokens.in"
       {"UTM_NORTHING", XT_UTM_NORTHING},
-#line 26 "xcsv_tokens.in"
-      {"GPS_HDOP", XT_GPS_HDOP},
 #line 27 "xcsv_tokens.in"
+      {"GPS_HDOP", XT_GPS_HDOP},
+#line 28 "xcsv_tokens.in"
       {"GPS_PDOP", XT_GPS_PDOP}
     };
 
@@ -300,11 +302,11 @@ Perfect_Hash::in_word_set (const char *str, size_t len)
       -1, -1, -1, -1, -1, -1, -1, -1, 70, -1, 71, -1, -1, -1,
       -1, -1, -1, -1, -1, 72, -1, -1, -1, -1, 73, -1, 74, -1,
       75, -1, 76, -1, 77, -1, 78, 79, 80, -1, -1, -1, -1, -1,
+      -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-      -1, -1, -1, -1, -1, -1, -1, 81, -1, -1, -1, -1, 82, -1,
-      -1, -1, -1, 83, 84, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 85
+      -1, -1, -1, -1, -1, -1, -1, 82, -1, -1, -1, -1, 83, -1,
+      -1, -1, -1, 84, 85, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+      -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 86
     };
 
   if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)

diff --git a/xcsv_tokens.in b/xcsv_tokens.in
index 5f3637d88eb187702c46e7435b8668c6a47a60a0..aced51e37ed052cb06abf6bf391b0a3927c495ca 100644 (file)
--- a/xcsv_tokens.in
+++ b/xcsv_tokens.in
@@ -8,6 +8,7 @@ CITY, XT_CITY
 CONSTANT, XT_CONSTANT
 COUNTRY, XT_COUNTRY
 DESCRIPTION, XT_DESCRIPTION
+EMAIL, XT_EMAIL
 EXCEL_TIME, XT_EXCEL_TIME
 FACILITY, XT_FACILITY
 FILENAME, XT_FILENAME

diff --git a/xmldoc/chapters/styles.xml b/xmldoc/chapters/styles.xml
index 5a4f8b43a960a06b5f91aadc33521a2876a55eb4..1fd535470eeee0db630319934b20054fb5eb7e93 100644 (file)
--- a/xmldoc/chapters/styles.xml
+++ b/xmldoc/chapters/styles.xml
@@ -1208,6 +1208,18 @@ longitude)
 
             <screen format="linespecific">COUNTRY, &quot;&quot;, &quot;%s&quot;</screen></para>
       </section>
+      <section id="style_email">
+         <title>EMAIL</title>
+         <para>An email address associated with a position.</para>
+         <para>example:
+
+
+
+
+
+
+            <screen format="linespecific">EMAIL, &quot;&quot;, &quot;%s&quot;</screen></para>
+      </section>
       <section id="style_facility">
          <title>FACILITY</title>
          <para>The name of a facility to associate with a position.</para>
@@ -1352,4 +1364,4 @@ look like:
    </section>
    <!-- notes -->
 </appendix>
-<!-- style -->
\ No newline at end of file
+<!-- style -->